package com.spider.moudle.crawler.forum;

import com.alibaba.fastjson.JSONObject;
import com.spider.moudle.api.Crawler;
import com.spider.moudle.api.Downloader;
import com.spider.moudle.api.LoginDownloader;
import com.spider.moudle.api.realize.AbstractCrawler;
import com.spider.moudle.api.realize.AbstractParamCrawler;
import com.spider.moudle.api.realize.AbstractReplyCrawler;
import com.spider.moudle.constant.Constant;
import com.spider.moudle.entity.*;
import com.spider.moudle.factory.DownloaderFactory;
import com.spider.moudle.factory.ExecutorServiceFactoy;
import com.spider.moudle.factory.NamedThreadFactory;
import com.spider.moudle.handler.DataHandler;
import com.spider.moudle.task.TaskManager;
import com.spider.moudle.util.CommonUtil;
import com.spider.moudle.util.DateUtil;
import com.spider.moudle.util.SettingsUtil;

import org.apache.http.Header;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.FutureTask;

/**
 * Created by Linbo Hu on 2017/12/18.
 * 爱卡社区抓取器
 */
public class AikaCrawler extends AbstractParamCrawler {

    private Setting setting = SettingsUtil.getSettingBySite("AiKa");

    private String index = setting.getIndex();

    private LoginDownloader downloader;

    private ArrayList<Cookie> cookies;

    @Override
    public LinkedList<Crawler> initial() {
        LinkedList<Crawler> list = new LinkedList<>();
        int count = setting.getCrawler_num();
        for (int i = 0; i < count; i++) {
            list.add(this.clone());
        }
        return list;
    }

    @Override
    protected AikaCrawler clone() {
        return new AikaCrawler(this.downloader, this.cookies);
    }

    /**
     * clone downloader
     * @param downloader
     * @param cookies
     */
    private AikaCrawler(LoginDownloader downloader, ArrayList<Cookie> cookies) {
        this.downloader = downloader;
        this.cookies = cookies;
    }

    public AikaCrawler() {
        downloader = getDownloader("http://www.xcar.com.cn/bbs/forumdisplay.php?fid=91");
    }

    private LoginDownloader getDownloader(String url) {
        for(int i = 0; i < 3; i++) {
            try {
                //访问北京论坛获取cookies,因为爱卡论坛需要cookies才能访问
                ResponseEntity entity = DownloaderFactory.buildDownloader("GBK").get(url);
                ArrayList<Header> headers = entity.getHeaders();
                cookies = Cookie.headersTranlateToCookie(headers);
                return DownloaderFactory.buildLoginDownloader(cookies, "GBK");
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return null;
    }

    @Override
    public void generateTask() {
        downloader = getDownloader("http://www.xcar.com.cn/bbs/forumdisplay.php?fid=91");
        ExecutorService executorService = null;
        try {
            ResponseEntity entity = null;
            try {
                try {
                    entity = downloader.get(index);
                } catch (Exception e) {
                    log.error(e.getMessage(), e);
                }
                if (entity != null && entity.getStatus_code() == 200) {
                    Document document = entity.getDocument();
                    Elements elements = document.select("div[id=\"carlife_bbs_box\"] dl dd a");
                    executorService = ExecutorServiceFactoy.buildExecutorService(setting.getTask_thread_count(), ExecutorServiceFactoy.getName(setting.getName(), setting.getIndex()), new NamedThreadFactory(setting.getName(), "生成任务"));
                    for (Element e : elements) {
                        String text = e.text();
                        if (text.contains("论坛")) {
                            text = text.replace("论坛", "");
                        }
                        Area area = Area.search(text);
                        String taskUrl = e.absUrl("href");
                        if(!taskUrl.contains("http")) {
                            continue;
                        }
                        AikaCrawler.AiKaTaskCrawler aiKaTaskCrawler = new AiKaTaskCrawler(taskUrl, area);
                        executorService.submit(aiKaTaskCrawler);
                    }
                }

            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (null != executorService) {
                executorService.shutdown();
                while (!executorService.isTerminated()) {
                    try {
                        Thread.sleep(1500);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }

    }

    @Override
    public JSONObject crawlItem(Task task) throws Exception {
        if (task.getTaskType() == TaskType.SPEECH) {
            JSONObject item = new JSONObject();
            SpeechBean data = new SpeechBean();
            String url = task.getUrl();
            data.setUrl(url);
            if (task.getArea() != null) {
                data.setArea(task.getArea());
            }
            ResponseEntity entity = null;
            try {
                entity = downloader.get(url);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (entity != null && entity.getStatus_code() == 200) {
                try {
                    Document document = entity.getDocument();
                    Elements msg_boxs = document.select(".t_row");
                    if (msg_boxs.isEmpty()) {
                        return null;
                    }
                    //解析发帖信息
                    Element publish_box = msg_boxs.first();
                    Elements publish_user_elements = publish_box.select(".t_user");
                    if (publish_user_elements.isEmpty()) {
                        return null;
                    }
                    Element publish_user_box = publish_user_elements.get(0);
                    String publish_user_name = publish_user_box.select("a[rel='nofollow']").first().text();
                    String publish_user_id = getUrlParams(publish_user_box.select("a[rel='nofollow']").first().attr
                            ("href")).get("uid").toString();
                    String publish_user_photo = publish_user_box.select(".avatar img").first().absUrl("src");
                    Element publish_msg_box = publish_box.select(".t_msg").first();
                    String publish_time = getTime(publish_msg_box.select("tbody > tr:nth-child(1) > td > div > " +
                            "div:nth-child(2)").text());
                    publish_time = publish_time + ":00";
                    publish_time = DateUtil.str_date_str(publish_time, DateUtil.yyyy_MM_dd_HH_mm_ss);
                    String content = publish_msg_box.select("tr").get(1).text();
                    List<String> img_urls = new ArrayList<>();
                    Elements imgElements1 = publish_msg_box.select("tr").get(1).select("img");
                    if (imgElements1.size() > 0) {
                        for (int imgIndex = 0; imgIndex < imgElements1.size(); imgIndex++) {
                            Element imgElement = imgElements1.get(imgIndex);
                            String img_url = imgElement.attr("src");
                            if (img_url.isEmpty()) {
                                continue;
                            }
                            if(!img_url.contains("http")) {
                                img_url = " http://www.xcar.com.cn//bbs/" + img_url;
                            }
                            img_urls.add(img_url);
                        }
                    }
                    String title = document.title();
                    String tid = getUrlParams(url).get("tid").toString();
                    Elements count_box_elements = document.select("#showPic");
                    if (count_box_elements.isEmpty()) {
                        return null;
                    }
                    Element count_box = count_box_elements.first();
                    String countString = count_box.text().replace("回复：", "").replace("查看：", "#");
                    String[] counts = countString.split("#");
                    int reply_num = Integer.parseInt(counts[0].replaceAll("\\s\\S", "").trim());
                    int read_num = Integer.parseInt(counts[1].replaceAll("\\s\\S", "").trim());
                    data.setReply_count(reply_num);
                    data.setRead_count(read_num);
                    data.setTitle(title);
                    data.setUrl(url);
                    data.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, publish_time));
                    data.setReproduce_count(0);
                    data.setLike_count(0);
                    data.setDislike_count(0);
                    data.setContent(CommonUtil.handle(content));
                    data.setImg_urls(img_urls);
                    data.setPublish_user_id(publish_user_id);
                    data.setPublish_user(publish_user_name);
                    data.setPublish_user_photo(publish_user_photo);
                    data.setPublish_user_jsonid(setting.getName() + "_" + publish_user_id);
                    data.setId(tid);
                    data.setSpider_time(new Date());

                    Element pageElement = document.select(".FpageNum").first();
                    Element lastPageElement = pageElement.select("a:contains(下一页)").first();
                    Element maxPageElement = lastPageElement.previousElementSibling();
                    int maxPage = Integer.parseInt(maxPageElement.text());
                    List<SpeechBean> reply_nodes = new ArrayList<>();
                    for (int i = 1; i < msg_boxs.size(); i++) {
                        SpeechBean reply = new SpeechBean();
                        Element reply_box = msg_boxs.get(i);
                        String msg_id = reply_box.attr("id");
                        msg_id = msg_id.split("_")[1];
                        String reply_url = url + "#pid" + msg_id;
                        Element reply_user_box = reply_box.select(".t_user").first();
                        String reply_user_name = reply_user_box.select("a[rel='nofollow']").first().text();
                        String reply_user_id = getUrlParams(reply_user_box.select("a[rel='nofollow']").first().attr
                                ("href")).get("uid").toString();
                        String reply_user_photo = reply_user_box.select(".avatar img").first().absUrl("src");
                        Element reply_msg_box = reply_box.select(".t_msg").first();
                        String reply_time = getTime(reply_msg_box.select(" tbody > tr:nth-child(1) > td > div > " +
                                "div:nth-child(2)").text().substring(4, 20));
                        String reply_content = reply_msg_box.select("tr").get(1).text();
                        List<String> reply_img_urls = new ArrayList<>();
                        Elements replyImgElements1 = reply_msg_box.select("tr").get(1).select("img");//图片
                        if (replyImgElements1.size() > 0) {
                            for (int imgIndex = 0; imgIndex < replyImgElements1.size(); imgIndex++) {
                                Element imgElement = replyImgElements1.get(imgIndex);
                                String img_url = imgElement.absUrl("src");
                                if (img_url.isEmpty()) {
                                    continue;
                                }
                                reply_img_urls.add(img_url);
                            }
                        }
                        reply.setId(msg_id);
                        reply.setPublish_user(reply_user_name);
                        reply.setPublish_user_id(reply_user_id);
                        reply.setPublish_user_photo(reply_user_photo);
                        reply.setContent(reply_content);
                        reply.setImg_urls(reply_img_urls);
                        reply.setPublish_user_jsonid(setting.getName() + "_" + reply_user_id);
                        reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, reply_time + ":00"));
                        reply.setUrl(reply_url);
                        reply.setParent_id(tid);
                        reply.setAncestor_id(tid);
                        reply.setArea(task.getArea());
                        if (CommonUtil.isNotNull(reply_user_id)) {
                            String userKey = setting.getName() + "_" + reply_user_id;
                            String userUrl = "http://a.xcar.com.cn/my/index.php?uid="+ reply_user_id;
                            Task hometask = new Task(userUrl, TaskType.USER, WebType.FORLUM, userKey);
                            hometask.put("publish_user", reply_user_name);
                            hometask.put("publish_user_photo", reply_user_photo);
                            TaskManager.add(setting.getUser_task(), hometask);
                        }
                        reply_nodes.add(reply);
                    }
                    //爬虫最大页码的限制，超过最大页的帖子就只抓到最大页码数
                    if (maxPage >= setting.getMax_page_count()) {
                        maxPage = setting.getMax_page_count();
                    }
                    //解析发帖信息
                    if (maxPage >= 2) {
                        int threadNum = setting.getReply_thread_count();
                        ExecutorService executorService = null;
                        try {
                            executorService = ExecutorServiceFactoy.buildExecutorService(threadNum, ExecutorServiceFactoy.getName(setting.getName(), url), new NamedThreadFactory(setting.getName(), "回复"));
                            List<FutureTask<List<SpeechBean>>> futureTasks = new ArrayList<>();
                            ArrayList<ArrayList<Integer>> splitPages = CommonUtil.splitPages(2, maxPage, threadNum);
                            for (int i = 0; i < splitPages.size(); i++) {
                                ReplyCrawler crawler = new ReplyCrawler(url, tid, tid, task.getArea(), splitPages.get(i));
                                FutureTask<List<SpeechBean>> futureTask = new FutureTask<>(crawler);
                                futureTasks.add(futureTask);
                                executorService.submit(futureTask);
                            }
                            for (FutureTask<List<SpeechBean>> futureTask : futureTasks) {
                                reply_nodes.addAll(futureTask.get());
                            }
                        } catch (Exception e) {
                            e.printStackTrace();
                        } finally {
                            if(executorService != null) {
                                executorService.shutdown();
                            }
                        }
                    }
                    data.setReply_nodes(reply_nodes);
                    item.put("data", data.toJSONObject());
                    item = dataHandler.handle(item, setting, TaskType.SPEECH);
                    return item;
                } catch (Exception e) {
                    e.printStackTrace();
                    log.error(setting.getName() + "获取言论数据解析错误: " + url);
                }
            }
        } else if (task.getTaskType() == TaskType.USER) {
            String userKey = task.getTaskKey();
            if (CommonUtil.isNull(userKey)) {
                System.err.println("User key is not empty where task type belong to home page");
                return null;
            }
            JSONObject item = new JSONObject();
            UserBean data;
            String url = task.getUrl();
            try {
                data = new UserBean();
                data.setJson_id(userKey);
                data.setUrl(url);
                String publish_user_id = userKey.substring(userKey.indexOf("_") + 1);
                data.setPublish_user_id(publish_user_id);
                String publish_user = (String) task.get("publish_user");
                data.setPublish_user(publish_user);
                String publish_user_photo = (String) task.get("publish_user_photo");
                data.setPublish_user_photo(publish_user_photo);
                ResponseEntity entity = null;
                try {
                    entity = downloader.get(url);
                } catch (Exception e) {
                    log.error(setting.getName() + "访问：" + url + "出错", e);
                }
                if (entity != null && entity.getStatus_code() == 200) {
                    Document document = entity.getDocument();

                    Elements info_con = document.select("div[class=\"info-con\"]");
                    if(!info_con.isEmpty()) {
                        Elements sex_elements = info_con.select(".sex");
                        if(!sex_elements.isEmpty()) {
                            Element sex_element = sex_elements.get(0);
                            String sex_string = sex_element.attr("class").split(" ")[1];
                            if("male".equals(sex_string)) {
                                data.setSex("男");
                            } else if("female".equals(sex_string)) {
                                data.setSex("女");
                            }
                        }

                        Elements register_elements = info_con.select("div[class=\"login-on\"] span");
                        if(!register_elements.isEmpty()) {
                            Element register_element = register_elements.get(0);
                            String register_string = register_element.text();
                            data.setRegister_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd, register_string));
                        }
                    }

                    Elements fan_follow_article_elements = document.select("div[class=\"info-num row\"] ul li");
                    if(!fan_follow_article_elements.isEmpty()) {
                        Element follow_element = fan_follow_article_elements.get(0).select("span").get(0);
                        String follow_count_string = follow_element.text();
                        int follow_count;
                        if(follow_count_string.endsWith("万")) {
                            follow_count_string = follow_count_string.replace("万", "");
                            if (follow_count_string.contains(".")) {
                                follow_count = Integer.parseInt(follow_count_string.replace(".", "")) * 1000;
                            } else {
                                follow_count = Integer.parseInt(follow_count_string) * 10000;
                            }
                        } else {
                            follow_count = Integer.parseInt(follow_count_string);
                        }
                        data.setFollow_count(follow_count);

                        Element fans_element = fan_follow_article_elements.get(1).select("span").get(0);
                        String fans_count_string = fans_element.text();
                        int fans_count;
                        if(fans_count_string.endsWith("万")) {
                            fans_count_string = fans_count_string.replace("万", "");
                            if(fans_count_string.contains(".")) {
                                fans_count = Integer.parseInt(fans_count_string.replace(".", "")) * 1000;
                            } else {
                                fans_count = Integer.parseInt(fans_count_string) * 10000;
                            }
                        } else {
                            fans_count = Integer.parseInt(fans_count_string);
                        }
                        data.setFans_count(fans_count);
                        Element article__element = fan_follow_article_elements.get(2).select("span").get(0);
                        String article_count_string = article__element.text();
                        int article_count;
                        if(article_count_string.endsWith("万")) {
                            article_count_string = article_count_string.replace("万", "");
                            if(article_count_string.contains(".")) {
                                article_count = Integer.parseInt(article_count_string.replace(".", "")) * 1000;
                            } else {
                                article_count = Integer.parseInt(article_count_string) * 10000;
                            }
                        } else {
                            article_count = Integer.parseInt(article_count_string);
                        }
                        data.setArticle_count(article_count);
                    }
                    item.put("data", data.toJSONObject());
                    item = dataHandler.handle(item, setting, TaskType.USER);
                    return item;
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return null;
    }

    /**
     * 抓取任务的子线程
     */
    public class AiKaTaskCrawler implements Runnable {

        private String url;

        private Area area;

        public AiKaTaskCrawler(String url, Area area) {
            this.url = url;
            this.area = area;
        }

        @Override
        public void run() {
            current(); //当前任务
        }

        private void current() {
            for (int page = 0; page < 5; page++) {
                //是否抓下一页
                boolean haveLater = false;
                ResponseEntity entity = null;
                try {
                    try {
                        entity = downloader.get(url);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entity != null && entity.getStatus_code() == 200) {
                        Document document = entity.getDocument();
                        Elements elements = document.select("dl[class=\"list_dl\"]");
                        for (int i = 0; i < elements.size(); i++) {
                            Element element = elements.get(i);
                            Elements aElements = element.select("a[class=\"titlink\"]");
                            if (aElements.isEmpty()) {
                                continue;
                            }
                            Element aElement = aElements.get(0);
                            String taskUrl = aElement.attr("href").trim();
                            if (!taskUrl.contains("http")) {
                                taskUrl = "http://www.xcar.com.cn/" + taskUrl;
                            }
                            String speechkey = taskUrl;
                            if (!speechkey.contains("=")) {
                                continue;
                            }
                            speechkey = speechkey.substring(speechkey.lastIndexOf("=") + 1, speechkey.length());
                            speechkey = setting.getSite() + "_" + speechkey;
                            Task task = new Task(taskUrl, TaskType.SPEECH, WebType.FORLUM, speechkey, area);
                            task.put("url", url);
                            TaskManager.add(setting.getSpeech_task(), task);
                            if (i == elements.size() - 1) {
                                Elements updatetimeElements = element.select("span[class=\"ttime\"]");
                                if (!updatetimeElements.isEmpty()) {
                                    String updatetimeString = updatetimeElements.get(0).text();
                                    Date now = new Date();
                                    Date updateTime = DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm, updatetimeString);
                                    if (updateTime.getTime() > now.getTime() - Constant.LAST_24_HOUR_TIME_STAMP) {
                                        Elements pageElements = document.select("a[class=\"page_down\"]");
                                        if (!pageElements.isEmpty()) {
                                            Element pageElement = pageElements.get(0);
                                            String laterUrl = pageElement.attr("href");
                                            if (!laterUrl.contains("http")) {
                                                laterUrl = "http://www.xcar.com.cn/" + laterUrl;
                                            }
                                            if (!url.equals(laterUrl)) {
                                                haveLater = true;
                                                url = laterUrl;
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                    //不存在下一页退出循环
                    if (!haveLater) {
                        break;
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }


    /**
     * 抓取回复的线程
     */
    private class ReplyCrawler extends AbstractReplyCrawler {

        private LoginDownloader downloader = getDownloader("http://www.xcar.com.cn/bbs/forumdisplay.php?fid=91");

        private ArrayList<Integer> pages;

        public ReplyCrawler(String url, String parentId, String ancestorId, Area area, ArrayList<Integer> pages) {
            this.url = url;
            this.parentId = parentId;
            this.ancestorId = ancestorId;
            this.area = area;
            this.pages = pages;
        }

        @Override
        public List<SpeechBean> call() throws Exception {
            List<SpeechBean> array = new ArrayList<>();
            for (int k = 0; k < pages.size(); k++) {
                String getUrl = url + "&page=" + pages.get(k);
                try {
                    ResponseEntity entity = null;
                    try {
                        entity = downloader.get(url);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entity != null && entity.getStatus_code() == 200) {
                        Document document = entity.getDocument();
                        Elements msg_boxs = document.select(".t_row");
                        for (int i = 0; i < msg_boxs.size(); i++) {
                            SpeechBean reply = new SpeechBean();
                            Element reply_box = msg_boxs.get(i);
                            String msg_id = reply_box.attr("id");
                            msg_id = msg_id.split("_")[1];
                            String reply_url = url + "#pid" + msg_id;
                            Element reply_user_box = reply_box.select(".t_user").first();
                            String reply_user_name = reply_user_box.select("a[rel='nofollow']").first().text();
                            String reply_user_id = getUrlParams(reply_user_box.select("a[rel='nofollow']").first()
                                    .attr("href")).get("uid").toString();
                            String reply_user_photo = reply_user_box.select(".avatar img").first().absUrl("src");
                            Element reply_msg_box = reply_box.select(".t_msg").first();
                            String reply_time = getTime(reply_msg_box.select(" tbody > tr:nth-child(1) > td > div > " + "div:nth-child(2)").text().substring(4, 20));
                            reply_time = reply_time + ":00";
                            String reply_content = reply_msg_box.select("tr").get(1).text();
                            List<String> reply_img_urls = new ArrayList<>();
                            Elements replyImgElements1 = reply_msg_box.select("tr").get(1).select("img");//图片
                            if (replyImgElements1.size() > 0) {
                                for (int imgIndex = 0; imgIndex < replyImgElements1.size(); imgIndex++) {
                                    Element imgElement = replyImgElements1.get(imgIndex);
                                    String img_url = imgElement.attr("src");
                                    if (img_url.isEmpty()) {
                                        continue;
                                    }
                                    if(!img_url.contains("http")) {
                                        img_url = " http://www.xcar.com.cn//bbs/" + img_url;
                                    }
                                    reply_img_urls.add(img_url);
                                }
                            }
                            reply.setId(msg_id);
                            reply.setPublish_user(reply_user_name);
                            reply.setPublish_user_id(reply_user_id);
                            reply.setPublish_user_photo(reply_user_photo);
                            reply.setContent(CommonUtil.handle(reply_content));
                            reply.setImg_urls(reply_img_urls);
                            reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, reply_time));
                            reply.setPublish_user_jsonid(setting.getName() + "_" + reply_user_id);
                            reply.setUrl(reply_url);
                            reply.setParent_id(parentId);
                            reply.setAncestor_id(ancestorId);
                            reply.setArea(area);
                            if (CommonUtil.isNotNull(reply_user_id)) {
                                String userKey = setting.getName() + "_" + reply_user_id;
                                String userUrl = "http://a.xcar.com.cn/my/index.php?uid="+ reply_user_id;
                                Task hometask = new Task(userUrl, TaskType.USER, WebType.FORLUM, userKey);
                                hometask.put("publish_user", reply_user_name);
                                hometask.put("publish_user_photo", reply_user_photo);
                                TaskManager.add(setting.getUser_task(), hometask);
                            }
                            array.add(reply);
                        }
                    }
                } catch (Exception e) {
                    log.error(e.getMessage(), e);
                }
            }
            return array;
        }
    }

    // TODO: 2017/12/27 爱卡社区测试
    public static void main(String[] args) throws Exception {
        System.out.println(new AikaCrawler().crawlItem(new Task("http://www.xcar.com.cn//bbs/viewthread.php?tid=11953748", TaskType.SPEECH, WebType.FORLUM, "ss", null)).toJSONString());
    }


}
